

<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<head>
  <meta charset="utf-8" />
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
  
  <title>mindspore.dataset.text &mdash; MindSpore master documentation</title>
  

  
  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />

  
  

  
  

  

  
  <!--[if lt IE 9]>
    <script src="../_static/js/html5shiv.min.js"></script>
  <![endif]-->
  
    
      <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
        <script src="../_static/jquery.js"></script>
        <script src="../_static/underscore.js"></script>
        <script src="../_static/doctools.js"></script>
        <script src="../_static/language_data.js"></script>
        <script async="async" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/latest.js?config=TeX-AMS-MML_HTMLorMML"></script>
    
    <script type="text/javascript" src="../_static/js/theme.js"></script>

    
    <link rel="index" title="Index" href="../genindex.html" />
    <link rel="search" title="Search" href="../search.html" />
    <link rel="next" title="mindspore.dataset.text.transforms.BasicTokenizer" href="dataset_text/mindspore.dataset.text.transforms.BasicTokenizer.html" />
    <link rel="prev" title="mindspore.dataset.config" href="mindspore.dataset.config.html" /> 
</head>

<body class="wy-body-for-nav">

   
  <div class="wy-grid-for-nav">
    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search" >
          

          
            <a href="../index.html" class="icon icon-home"> MindSpore
          

          
          </a>

          
            
            
          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        
        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
              
            
            
              <p class="caption"><span class="caption-text">MindSpore Python API</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="mindspore.html">mindspore</a></li>
<li class="toctree-l1"><a class="reference internal" href="mindspore.common.initializer.html">mindspore.common.initializer</a></li>
<li class="toctree-l1"><a class="reference internal" href="mindspore.communication.html">mindspore.communication</a></li>
<li class="toctree-l1"><a class="reference internal" href="mindspore.compression.html">mindspore.compression</a></li>
<li class="toctree-l1"><a class="reference internal" href="mindspore.context.html">mindspore.context</a></li>
<li class="toctree-l1"><a class="reference internal" href="mindspore.dataset.html">mindspore.dataset</a></li>
<li class="toctree-l1"><a class="reference internal" href="mindspore.dataset.audio.html">mindspore.dataset.audio</a></li>
<li class="toctree-l1"><a class="reference internal" href="mindspore.dataset.config.html">mindspore.dataset.config</a></li>
<li class="toctree-l1 current"><a class="current reference internal" href="#">mindspore.dataset.text</a><ul>
<li class="toctree-l2"><a class="reference internal" href="#mindspore-dataset-text-transforms">mindspore.dataset.text.transforms</a><ul>
<li class="toctree-l3"><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.BasicTokenizer.html">mindspore.dataset.text.transforms.BasicTokenizer</a></li>
<li class="toctree-l3"><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.BertTokenizer.html">mindspore.dataset.text.transforms.BertTokenizer</a></li>
<li class="toctree-l3"><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.CaseFold.html">mindspore.dataset.text.transforms.CaseFold</a></li>
<li class="toctree-l3"><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.JiebaTokenizer.html">mindspore.dataset.text.transforms.JiebaTokenizer</a></li>
<li class="toctree-l3"><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.Lookup.html">mindspore.dataset.text.transforms.Lookup</a></li>
<li class="toctree-l3"><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.Ngram.html">mindspore.dataset.text.transforms.Ngram</a></li>
<li class="toctree-l3"><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.NormalizeUTF8.html">mindspore.dataset.text.transforms.NormalizeUTF8</a></li>
<li class="toctree-l3"><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.PythonTokenizer.html">mindspore.dataset.text.transforms.PythonTokenizer</a></li>
<li class="toctree-l3"><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.RegexReplace.html">mindspore.dataset.text.transforms.RegexReplace</a></li>
<li class="toctree-l3"><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.RegexTokenizer.html">mindspore.dataset.text.transforms.RegexTokenizer</a></li>
<li class="toctree-l3"><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.SentencePieceTokenizer.html">mindspore.dataset.text.transforms.SentencePieceTokenizer</a></li>
<li class="toctree-l3"><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.SlidingWindow.html">mindspore.dataset.text.transforms.SlidingWindow</a></li>
<li class="toctree-l3"><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.ToNumber.html">mindspore.dataset.text.transforms.ToNumber</a></li>
<li class="toctree-l3"><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.TruncateSequencePair.html">mindspore.dataset.text.transforms.TruncateSequencePair</a></li>
<li class="toctree-l3"><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.UnicodeCharTokenizer.html">mindspore.dataset.text.transforms.UnicodeCharTokenizer</a></li>
<li class="toctree-l3"><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.UnicodeScriptTokenizer.html">mindspore.dataset.text.transforms.UnicodeScriptTokenizer</a></li>
<li class="toctree-l3"><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.WhitespaceTokenizer.html">mindspore.dataset.text.transforms.WhitespaceTokenizer</a></li>
<li class="toctree-l3"><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.WordpieceTokenizer.html">mindspore.dataset.text.transforms.WordpieceTokenizer</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="#mindspore-dataset-text-utils">mindspore.dataset.text.utils</a><ul>
<li class="toctree-l3"><a class="reference internal" href="dataset_text/mindspore.dataset.text.JiebaMode.html">mindspore.dataset.text.JiebaMode</a></li>
<li class="toctree-l3"><a class="reference internal" href="dataset_text/mindspore.dataset.text.NormalizeForm.html">mindspore.dataset.text.NormalizeForm</a></li>
<li class="toctree-l3"><a class="reference internal" href="dataset_text/mindspore.dataset.text.SentencePieceModel.html">mindspore.dataset.text.SentencePieceModel</a></li>
<li class="toctree-l3"><a class="reference internal" href="dataset_text/mindspore.dataset.text.SentencePieceVocab.html">mindspore.dataset.text.SentencePieceVocab</a></li>
<li class="toctree-l3"><a class="reference internal" href="dataset_text/mindspore.dataset.text.SPieceTokenizerLoadType.html">mindspore.dataset.text.SPieceTokenizerLoadType</a></li>
<li class="toctree-l3"><a class="reference internal" href="dataset_text/mindspore.dataset.text.SPieceTokenizerOutType.html">mindspore.dataset.text.SPieceTokenizerOutType</a></li>
<li class="toctree-l3"><a class="reference internal" href="dataset_text/mindspore.dataset.text.to_str.html">mindspore.dataset.text.to_str</a></li>
<li class="toctree-l3"><a class="reference internal" href="dataset_text/mindspore.dataset.text.to_bytes.html">mindspore.dataset.text.to_bytes</a></li>
<li class="toctree-l3"><a class="reference internal" href="dataset_text/mindspore.dataset.text.Vocab.html">mindspore.dataset.text.Vocab</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="mindspore.dataset.transforms.html">mindspore.dataset.transforms</a></li>
<li class="toctree-l1"><a class="reference internal" href="mindspore.dataset.vision.html">mindspore.dataset.vision</a></li>
<li class="toctree-l1"><a class="reference internal" href="mindspore.mindrecord.html">mindspore.mindrecord</a></li>
<li class="toctree-l1"><a class="reference internal" href="mindspore.nn.html">mindspore.nn</a></li>
<li class="toctree-l1"><a class="reference internal" href="mindspore.nn.probability.html">mindspore.nn.probability</a></li>
<li class="toctree-l1"><a class="reference internal" href="mindspore.nn.transformer.html">mindspore.nn.transformer</a></li>
<li class="toctree-l1"><a class="reference internal" href="mindspore.numpy.html">mindspore.numpy</a></li>
<li class="toctree-l1"><a class="reference internal" href="mindspore.ops.html">mindspore.ops</a></li>
<li class="toctree-l1"><a class="reference internal" href="mindspore.parallel.html">mindspore.parallel</a></li>
<li class="toctree-l1"><a class="reference internal" href="mindspore.parallel.nn.html">mindspore.parallel.nn</a></li>
<li class="toctree-l1"><a class="reference internal" href="mindspore.profiler.html">mindspore.profiler</a></li>
<li class="toctree-l1"><a class="reference internal" href="mindspore.scipy.html">mindspore.scipy</a></li>
<li class="toctree-l1"><a class="reference internal" href="mindspore.train.html">mindspore.train</a></li>
<li class="toctree-l1"><a class="reference internal" href="mindspore.boost.html">mindspore.boost</a></li>
</ul>
<p class="caption"><span class="caption-text">MindSpore C++ API</span></p>
<ul>
<li class="toctree-l1"><a class="reference external" href="https://www.mindspore.cn/lite/api/zh-CN/master/api_cpp/mindspore.html">MindSpore Lite↗</a></li>
</ul>

            
          
        </div>
        
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" aria-label="top navigation">
        
          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
          <a href="../index.html">MindSpore</a>
        
      </nav>


      <div class="wy-nav-content">
        
        <div class="rst-content">
        
          

















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="../index.html" class="icon icon-home"></a> &raquo;</li>
        
      <li>mindspore.dataset.text</li>
    
    
      <li class="wy-breadcrumbs-aside">
        
          
            <a href="../_sources/api_python/mindspore.dataset.text.rst.txt" rel="nofollow"> View page source</a>
          
        
      </li>
    
  </ul>

  
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <div class="section" id="mindspore-dataset-text">
<h1>mindspore.dataset.text<a class="headerlink" href="#mindspore-dataset-text" title="Permalink to this headline">¶</a></h1>
<p>此模块用于文本数据增强，包括 <cite>transforms</cite> 和 <cite>utils</cite> 两个子模块。</p>
<p><cite>transforms</cite> 是一个高性能文本数据增强模块，支持常见的文本数据增强处理。</p>
<p><cite>utils</cite> 提供了一些文本处理的工具方法。</p>
<p>在API示例中，常用的模块导入方法如下：</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">mindspore.dataset</span> <span class="k">as</span> <span class="nn">ds</span>
<span class="kn">from</span> <span class="nn">mindspore.dataset</span> <span class="kn">import</span> <span class="n">text</span>
</pre></div>
</div>
<div class="section" id="mindspore-dataset-text-transforms">
<h2>mindspore.dataset.text.transforms<a class="headerlink" href="#mindspore-dataset-text-transforms" title="Permalink to this headline">¶</a></h2>
<table class="longtable docutils align-default">
<colgroup>
<col style="width: 10%" />
<col style="width: 60%" />
<col style="width: 30%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p>接口名</p></td>
<td><p>概述</p></td>
<td><p>说明</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.BasicTokenizer.html#mindspore.dataset.text.transforms.BasicTokenizer" title="mindspore.dataset.text.transforms.BasicTokenizer"><code class="xref py py-obj docutils literal notranslate"><span class="pre">mindspore.dataset.text.transforms.BasicTokenizer</span></code></a></p></td>
<td><p>通过特定规则标记UTF-8字符串的标量Tensor。</p></td>
<td><p>Windows平台尚不支持BasicTokenizer</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.BertTokenizer.html#mindspore.dataset.text.transforms.BertTokenizer" title="mindspore.dataset.text.transforms.BertTokenizer"><code class="xref py py-obj docutils literal notranslate"><span class="pre">mindspore.dataset.text.transforms.BertTokenizer</span></code></a></p></td>
<td><p>Tokenizer used for Bert text process.</p></td>
<td><p></p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.CaseFold.html#mindspore.dataset.text.transforms.CaseFold" title="mindspore.dataset.text.transforms.CaseFold"><code class="xref py py-obj docutils literal notranslate"><span class="pre">mindspore.dataset.text.transforms.CaseFold</span></code></a></p></td>
<td><p>对UTF-8字符串进行大小写转换，相比 <code class="xref py py-func docutils literal notranslate"><span class="pre">str.lower()</span></code> 支持更多字符。</p></td>
<td><p>Windows 平台尚不支持 CaseFold</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.JiebaTokenizer.html#mindspore.dataset.text.transforms.JiebaTokenizer" title="mindspore.dataset.text.transforms.JiebaTokenizer"><code class="xref py py-obj docutils literal notranslate"><span class="pre">mindspore.dataset.text.transforms.JiebaTokenizer</span></code></a></p></td>
<td><p>使用Jieba分词器对中文字符串进行分词。</p></td>
<td><p>必须保证 HMMSEgment 算法和 MPSegment 算法所使用的字典文件的完整性</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.Lookup.html#mindspore.dataset.text.transforms.Lookup" title="mindspore.dataset.text.transforms.Lookup"><code class="xref py py-obj docutils literal notranslate"><span class="pre">mindspore.dataset.text.transforms.Lookup</span></code></a></p></td>
<td><p>根据词表，将分词标记(token)映射到其索引值(id)。</p></td>
<td><p></p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.Ngram.html#mindspore.dataset.text.transforms.Ngram" title="mindspore.dataset.text.transforms.Ngram"><code class="xref py py-obj docutils literal notranslate"><span class="pre">mindspore.dataset.text.transforms.Ngram</span></code></a></p></td>
<td><p>从1-D的字符串生成N-gram。</p></td>
<td><p></p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.NormalizeUTF8.html#mindspore.dataset.text.transforms.NormalizeUTF8" title="mindspore.dataset.text.transforms.NormalizeUTF8"><code class="xref py py-obj docutils literal notranslate"><span class="pre">mindspore.dataset.text.transforms.NormalizeUTF8</span></code></a></p></td>
<td><p>对UTF-8编码的字符串进行规范化处理。</p></td>
<td><p>Windows 平台尚不支持 NormalizeUTF8</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.PythonTokenizer.html#mindspore.dataset.text.transforms.PythonTokenizer" title="mindspore.dataset.text.transforms.PythonTokenizer"><code class="xref py py-obj docutils literal notranslate"><span class="pre">mindspore.dataset.text.transforms.PythonTokenizer</span></code></a></p></td>
<td><p>使用用户自定义的分词器对输入字符串进行分词。</p></td>
<td><p></p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.RegexReplace.html#mindspore.dataset.text.transforms.RegexReplace" title="mindspore.dataset.text.transforms.RegexReplace"><code class="xref py py-obj docutils literal notranslate"><span class="pre">mindspore.dataset.text.transforms.RegexReplace</span></code></a></p></td>
<td><p>根据正则表达式对UTF-8编码格式的字符串内容进行正则替换。</p></td>
<td><p>Windows 平台尚不支持 RegexReplace</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.RegexTokenizer.html#mindspore.dataset.text.transforms.RegexTokenizer" title="mindspore.dataset.text.transforms.RegexTokenizer"><code class="xref py py-obj docutils literal notranslate"><span class="pre">mindspore.dataset.text.transforms.RegexTokenizer</span></code></a></p></td>
<td><p>根据正则表达式对字符串进行分词。</p></td>
<td><p>Windows 平台尚不支持 RegexTokenizer</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.SentencePieceTokenizer.html#mindspore.dataset.text.transforms.SentencePieceTokenizer" title="mindspore.dataset.text.transforms.SentencePieceTokenizer"><code class="xref py py-obj docutils literal notranslate"><span class="pre">mindspore.dataset.text.transforms.SentencePieceTokenizer</span></code></a></p></td>
<td><p>使用SentencePiece分词器对字符串进行分词。</p></td>
<td><p></p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.SlidingWindow.html#mindspore.dataset.text.transforms.SlidingWindow" title="mindspore.dataset.text.transforms.SlidingWindow"><code class="xref py py-obj docutils literal notranslate"><span class="pre">mindspore.dataset.text.transforms.SlidingWindow</span></code></a></p></td>
<td><p>在输入数据的某个维度上进行滑窗切分处理。</p></td>
<td><p></p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.ToNumber.html#mindspore.dataset.text.transforms.ToNumber" title="mindspore.dataset.text.transforms.ToNumber"><code class="xref py py-obj docutils literal notranslate"><span class="pre">mindspore.dataset.text.transforms.ToNumber</span></code></a></p></td>
<td><p>将字符串的每个元素转换为数字。</p></td>
<td><p></p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.TruncateSequencePair.html#mindspore.dataset.text.transforms.TruncateSequencePair" title="mindspore.dataset.text.transforms.TruncateSequencePair"><code class="xref py py-obj docutils literal notranslate"><span class="pre">mindspore.dataset.text.transforms.TruncateSequencePair</span></code></a></p></td>
<td><p>截断一对 1-D 字符串的内容，使其总长度小于给定长度。</p></td>
<td><p></p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.UnicodeCharTokenizer.html#mindspore.dataset.text.transforms.UnicodeCharTokenizer" title="mindspore.dataset.text.transforms.UnicodeCharTokenizer"><code class="xref py py-obj docutils literal notranslate"><span class="pre">mindspore.dataset.text.transforms.UnicodeCharTokenizer</span></code></a></p></td>
<td><p>使用Unicode分词器将字符串分词为Unicode字符。</p></td>
<td><p></p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.UnicodeScriptTokenizer.html#mindspore.dataset.text.transforms.UnicodeScriptTokenizer" title="mindspore.dataset.text.transforms.UnicodeScriptTokenizer"><code class="xref py py-obj docutils literal notranslate"><span class="pre">mindspore.dataset.text.transforms.UnicodeScriptTokenizer</span></code></a></p></td>
<td><p>使用UnicodeScript分词器对字符串进行分词。</p></td>
<td><p>Windows 平台尚不支持 UnicodeScriptTokenizer</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.WhitespaceTokenizer.html#mindspore.dataset.text.transforms.WhitespaceTokenizer" title="mindspore.dataset.text.transforms.WhitespaceTokenizer"><code class="xref py py-obj docutils literal notranslate"><span class="pre">mindspore.dataset.text.transforms.WhitespaceTokenizer</span></code></a></p></td>
<td><p>基于ICU4C定义的空白字符（’ ‘, ‘\t’, ‘\r’, ‘\n’）对输入字符串进行分词。</p></td>
<td><p>Windows 平台尚不支持 WhitespaceTokenizer</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="dataset_text/mindspore.dataset.text.transforms.WordpieceTokenizer.html#mindspore.dataset.text.transforms.WordpieceTokenizer" title="mindspore.dataset.text.transforms.WordpieceTokenizer"><code class="xref py py-obj docutils literal notranslate"><span class="pre">mindspore.dataset.text.transforms.WordpieceTokenizer</span></code></a></p></td>
<td><p>对输入的1-D字符串分词为子词(subword)标记。</p></td>
<td><p></p></td>
</tr>
</tbody>
</table>
</div>
<div class="section" id="mindspore-dataset-text-utils">
<h2>mindspore.dataset.text.utils<a class="headerlink" href="#mindspore-dataset-text-utils" title="Permalink to this headline">¶</a></h2>
<table class="longtable docutils align-default">
<colgroup>
<col style="width: 10%" />
<col style="width: 60%" />
<col style="width: 30%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p>接口名</p></td>
<td><p>概述</p></td>
<td><p>说明</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="dataset_text/mindspore.dataset.text.JiebaMode.html#mindspore.dataset.text.JiebaMode" title="mindspore.dataset.text.JiebaMode"><code class="xref py py-obj docutils literal notranslate"><span class="pre">mindspore.dataset.text.JiebaMode</span></code></a></p></td>
<td><p><code class="xref py py-class docutils literal notranslate"><span class="pre">JiebaTokenizer</span></code> 的枚举值。</p></td>
<td><p></p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="dataset_text/mindspore.dataset.text.NormalizeForm.html#mindspore.dataset.text.NormalizeForm" title="mindspore.dataset.text.NormalizeForm"><code class="xref py py-obj docutils literal notranslate"><span class="pre">mindspore.dataset.text.NormalizeForm</span></code></a></p></td>
<td><p><code class="xref py py-class docutils literal notranslate"><span class="pre">NormalizeUTF8</span></code> 的枚举值。</p></td>
<td><p></p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="dataset_text/mindspore.dataset.text.SentencePieceModel.html#mindspore.dataset.text.SentencePieceModel" title="mindspore.dataset.text.SentencePieceModel"><code class="xref py py-obj docutils literal notranslate"><span class="pre">mindspore.dataset.text.SentencePieceModel</span></code></a></p></td>
<td><p><cite>SentencePiece</cite> 分词方法的枚举类。</p></td>
<td><p></p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="dataset_text/mindspore.dataset.text.SentencePieceVocab.html#mindspore.dataset.text.SentencePieceVocab" title="mindspore.dataset.text.SentencePieceVocab"><code class="xref py py-obj docutils literal notranslate"><span class="pre">mindspore.dataset.text.SentencePieceVocab</span></code></a></p></td>
<td><p>用于执行分词的SentencePiece对象。</p></td>
<td><p></p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="dataset_text/mindspore.dataset.text.SPieceTokenizerLoadType.html#mindspore.dataset.text.SPieceTokenizerLoadType" title="mindspore.dataset.text.SPieceTokenizerLoadType"><code class="xref py py-obj docutils literal notranslate"><span class="pre">mindspore.dataset.text.SPieceTokenizerLoadType</span></code></a></p></td>
<td><p><code class="xref py py-class docutils literal notranslate"><span class="pre">SentencePieceTokenizer</span></code> 加载类型的枚举值。</p></td>
<td><p></p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="dataset_text/mindspore.dataset.text.SPieceTokenizerOutType.html#mindspore.dataset.text.SPieceTokenizerOutType" title="mindspore.dataset.text.SPieceTokenizerOutType"><code class="xref py py-obj docutils literal notranslate"><span class="pre">mindspore.dataset.text.SPieceTokenizerOutType</span></code></a></p></td>
<td><p><code class="xref py py-class docutils literal notranslate"><span class="pre">SentencePieceTokenizer</span></code> 输出类型的枚举值。</p></td>
<td><p></p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="dataset_text/mindspore.dataset.text.to_str.html#mindspore.dataset.text.to_str" title="mindspore.dataset.text.to_str"><code class="xref py py-obj docutils literal notranslate"><span class="pre">mindspore.dataset.text.to_str</span></code></a></p></td>
<td><p>基于 <cite>encoding</cite> 字符集对每个元素进行解码，借此将 <cite>bytes</cite> 的NumPy数组转换为 <cite>string</cite> 的数组。</p></td>
<td><p></p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="dataset_text/mindspore.dataset.text.to_bytes.html#mindspore.dataset.text.to_bytes" title="mindspore.dataset.text.to_bytes"><code class="xref py py-obj docutils literal notranslate"><span class="pre">mindspore.dataset.text.to_bytes</span></code></a></p></td>
<td><p>基于 <cite>encoding</cite> 字符集对每个元素进行编码，将 <cite>string</cite> 的NumPy数组转换为 <cite>bytes</cite> 的数组。</p></td>
<td><p></p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="dataset_text/mindspore.dataset.text.Vocab.html#mindspore.dataset.text.Vocab" title="mindspore.dataset.text.Vocab"><code class="xref py py-obj docutils literal notranslate"><span class="pre">mindspore.dataset.text.Vocab</span></code></a></p></td>
<td><p>用于查找单词的vocab对象。</p></td>
<td><p></p></td>
</tr>
</tbody>
</table>
</div>
</div>


           </div>
           
          </div>
          <footer>
    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
        <a href="dataset_text/mindspore.dataset.text.transforms.BasicTokenizer.html" class="btn btn-neutral float-right" title="mindspore.dataset.text.transforms.BasicTokenizer" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
        <a href="mindspore.dataset.config.html" class="btn btn-neutral float-left" title="mindspore.dataset.config" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
    </div>

  <hr/>

  <div role="contentinfo">
    <p>
        &#169; Copyright 2021, MindSpore.

    </p>
  </div>
    
    
    
    Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
    
    <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
    
    provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>
        </div>
      </div>

    </section>

  </div>
  

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>

  
  
    
   

</body>
</html>